{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ZMPanjastOY4"
   },
   "outputs": [],
   "source": [
    "#Installing FinRL\n",
    "%%capture\n",
    "!pip install wrds\n",
    "!pip install swig\n",
    "!pip install git+https://github.com/AI4Finance-Foundation/FinRL.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "W_0SxBYTtWNB"
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip install \"ray[tune]\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "iY2GccjQdmKu"
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip install GPy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "lPh7bRBVL9u3"
   },
   "source": [
    "#Importing libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "AnFm0-vntYQw",
    "outputId": "ad545740-d650-442e-df9c-3b8c1f496587"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/pyfolio/pos.py:27: UserWarning: Module \"zipline.assets\" not found; multipliers will not be applied to position notionals.\n",
      "  'Module \"zipline.assets\" not found; multipliers will not be applied'\n"
     ]
    }
   ],
   "source": [
    "#Importing the libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "# matplotlib.use('Agg')\n",
    "import datetime\n",
    "%matplotlib inline\n",
    "# from finrl import config\n",
    "import finrl\n",
    "# from finrl import meta\n",
    "# from finrl import applications\n",
    "# from finrl import agents\n",
    "from finrl import config, config_tickers\n",
    "from finrl.meta.preprocessor.yahoodownloader import YahooDownloader\n",
    "from finrl.meta.preprocessor.preprocessors import FeatureEngineer, data_split\n",
    "from finrl.meta.env_stock_trading.env_stocktrading_np import StockTradingEnv as StockTradingEnv_numpy \n",
    "from finrl.meta.env_stock_trading.env_stocktrading import StockTradingEnv\n",
    "from finrl.agents.rllib.models import DRLAgent as DRLAgent_rllib\n",
    "from finrl.meta.data_processor import DataProcessor\n",
    "# from stable_baselines3.common.vec_env import DummyVecEnv\n",
    "from finrl.plot import backtest_stats, backtest_plot, get_daily_return, get_baseline\n",
    "import ray\n",
    "from pprint import pprint\n",
    "from ray.rllib.agents.ppo import PPOTrainer\n",
    "from ray.rllib.agents.ddpg import DDPGTrainer\n",
    "from ray.rllib.agents.a3c.a2c import A2CTrainer\n",
    "from ray.rllib.agents.a2c import a2c\n",
    "from ray.rllib.algorithms.ddpg import ddpg, td3\n",
    "from ray.rllib.algorithms.ppo import ppo\n",
    "from ray.rllib.algorithms.sac import sac\n",
    "from ray.tune.schedulers import PopulationBasedTraining\n",
    "from ray.tune.schedulers.pb2 import PB2\n",
    "from ray.tune import run, sample_from\n",
    "from ray.tune.registry import register_env\n",
    "\n",
    "import sys\n",
    "import os\n",
    "import itertools\n",
    "from ray import tune\n",
    "from ray.tune.suggest import ConcurrencyLimiter\n",
    "\n",
    "from ray.tune.registry import register_env\n",
    "\n",
    "import time\n",
    "from typing import Dict, Optional, Any\n",
    "import psutil\n",
    "psutil_memory_in_bytes = psutil.virtual_memory().total\n",
    "\n",
    "ray._private.utils.get_system_memory = lambda: psutil_memory_in_bytes\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Yb3PMaAZ2gUy"
   },
   "outputs": [],
   "source": [
    "MODELS = {\"a2c\": a2c, \"ddpg\": ddpg, \"td3\": td3, \"sac\": sac, \"ppo\": ppo}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ZWG4u7NsOI98"
   },
   "source": [
    "## Helper functions for train and test environments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Sx6O3qevuaDC"
   },
   "outputs": [],
   "source": [
    "#Function to calculate the sharpe ratio from the list of total_episode_reward\n",
    "def calculate_sharpe(episode_reward:list):\n",
    "  perf_data = pd.DataFrame(data=episode_reward,columns=['reward'])\n",
    "  perf_data['daily_return'] = perf_data['reward'].pct_change(1)\n",
    "  if perf_data['daily_return'].std() !=0:\n",
    "    sharpe = (252**0.5)*perf_data['daily_return'].mean()/ \\\n",
    "          perf_data['daily_return'].std()\n",
    "    return sharpe\n",
    "  else:\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HM12Fz7IrN4P"
   },
   "outputs": [],
   "source": [
    "TRAIN_START_DATE = '2014-01-01'\n",
    "TRAIN_END_DATE = '2019-07-30'\n",
    "\n",
    "VAL_START_DATE = '2019-08-01'\n",
    "VAL_END_DATE = '2020-07-30'\n",
    "\n",
    "TEST_START_DATE = '2020-08-01'\n",
    "TEST_END_DATE = '2021-10-01'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "tjrZFwhLsHHH"
   },
   "outputs": [],
   "source": [
    "from finrl.config_tickers import DOW_30_TICKER\n",
    "from config import INDICATORS\n",
    "technical_indicator_list = INDICATORS\n",
    "\n",
    "model_name = 'ppo'\n",
    "env = StockTradingEnv_numpy\n",
    "# ticker_list = ['SPY','TSLA','AAPL','GOOGL']\n",
    "ticker_list = DOW_30_TICKER\n",
    "data_source = 'yahoofinance'\n",
    "time_interval = '1D'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zbCRpstzRz5y"
   },
   "outputs": [],
   "source": [
    "def get_train_env(start_date, end_date, ticker_list, data_source, time_interval, \n",
    "          technical_indicator_list, env, model_name, if_vix = True,\n",
    "          **kwargs):\n",
    "    \n",
    "    #fetch data\n",
    "    DP = DataProcessor(data_source, **kwargs)\n",
    "    data = DP.download_data(ticker_list, start_date, end_date, time_interval)\n",
    "    data = DP.clean_data(data)\n",
    "    data = DP.add_technical_indicator(data, technical_indicator_list)\n",
    "    if if_vix:\n",
    "        data = DP.add_vix(data)\n",
    "    price_array, tech_array, turbulence_array = DP.df_to_array(data, if_vix)\n",
    "    train_env_config = {'price_array':price_array,\n",
    "              'tech_array':tech_array,\n",
    "              'turbulence_array':turbulence_array,\n",
    "              'if_train':True}\n",
    "    \n",
    "    return train_env_config\n",
    "\n",
    "def calculate_sharpe(episode_reward:list):\n",
    "  perf_data = pd.DataFrame(data=episode_reward,columns=['reward'])\n",
    "  perf_data['daily_return'] = perf_data['reward'].pct_change(1)\n",
    "  if perf_data['daily_return'].std() !=0:\n",
    "    sharpe = (252**0.5)*perf_data['daily_return'].mean()/ \\\n",
    "          perf_data['daily_return'].std()\n",
    "    return sharpe\n",
    "  else:\n",
    "    return 0\n",
    "\n",
    "def get_test_config(start_date, end_date, ticker_list, data_source, time_interval, \n",
    "         technical_indicator_list, env, model_name, if_vix = True,\n",
    "         **kwargs):\n",
    "  \n",
    "  DP = DataProcessor(data_source, **kwargs)\n",
    "  data = DP.download_data(ticker_list, start_date, end_date, time_interval)\n",
    "  data = DP.clean_data(data)\n",
    "  data = DP.add_technical_indicator(data, technical_indicator_list)\n",
    "  \n",
    "  if if_vix:\n",
    "      data = DP.add_vix(data)\n",
    "  \n",
    "  price_array, tech_array, turbulence_array = DP.df_to_array(data, if_vix)\n",
    "  test_env_config = {'price_array':price_array,\n",
    "            'tech_array':tech_array,\n",
    "            'turbulence_array':turbulence_array,'if_train':False}\n",
    "  return test_env_config\n",
    "\n",
    "def val_or_test(test_env_config,agent_path,model_name,env):\n",
    "  episode_total_reward = DRL_prediction(model_name,test_env_config,\n",
    "                                env = env,\n",
    "                                agent_path=agent_path)\n",
    "\n",
    "\n",
    "  return calculate_sharpe(episode_total_reward),episode_total_reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3Egt3KdmGght",
    "outputId": "57ad37aa-c95f-4581-f8d8-c9a5a540aef6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "Shape of DataFrame:  (48339, 9)\n",
      "Clean data for AAPL\n",
      "Data clean for AAPL is finished.\n",
      "Clean data for AMGN\n",
      "Data clean for AMGN is finished.\n",
      "Clean data for AXP\n",
      "Data clean for AXP is finished.\n",
      "Clean data for BA\n",
      "Data clean for BA is finished.\n",
      "Clean data for CAT\n",
      "Data clean for CAT is finished.\n",
      "Clean data for CRM\n",
      "Data clean for CRM is finished.\n",
      "Clean data for CSCO\n",
      "Data clean for CSCO is finished.\n",
      "Clean data for CVX\n",
      "Data clean for CVX is finished.\n",
      "Clean data for DIS\n",
      "Data clean for DIS is finished.\n",
      "Clean data for DOW\n",
      "NaN data on start date, fill using first valid data.\n",
      "Data clean for DOW is finished.\n",
      "Clean data for GS\n",
      "Data clean for GS is finished.\n",
      "Clean data for HD\n",
      "Data clean for HD is finished.\n",
      "Clean data for HON\n",
      "Data clean for HON is finished.\n",
      "Clean data for IBM\n",
      "Data clean for IBM is finished.\n",
      "Clean data for INTC\n",
      "Data clean for INTC is finished.\n",
      "Clean data for JNJ\n",
      "Data clean for JNJ is finished.\n",
      "Clean data for JPM\n",
      "Data clean for JPM is finished.\n",
      "Clean data for KO\n",
      "Data clean for KO is finished.\n",
      "Clean data for MCD\n",
      "Data clean for MCD is finished.\n",
      "Clean data for MMM\n",
      "Data clean for MMM is finished.\n",
      "Clean data for MRK\n",
      "Data clean for MRK is finished.\n",
      "Clean data for MSFT\n",
      "Data clean for MSFT is finished.\n",
      "Clean data for NKE\n",
      "Data clean for NKE is finished.\n",
      "Clean data for PG\n",
      "Data clean for PG is finished.\n",
      "Clean data for TRV\n",
      "Data clean for TRV is finished.\n",
      "Clean data for UNH\n",
      "Data clean for UNH is finished.\n",
      "Clean data for V\n",
      "Data clean for V is finished.\n",
      "Clean data for VZ\n",
      "Data clean for VZ is finished.\n",
      "Clean data for WBA\n",
      "Data clean for WBA is finished.\n",
      "Clean data for WMT\n",
      "Data clean for WMT is finished.\n",
      "Data clean all finished!\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "Shape of DataFrame:  (1655, 9)\n",
      "Clean data for ^VIX\n",
      "Data clean for ^VIX is finished.\n",
      "Data clean all finished!\n",
      "['AAPL' 'AMGN' 'AXP' 'BA' 'CAT' 'CRM' 'CSCO' 'CVX' 'DIS' 'DOW' 'GS' 'HD'\n",
      " 'HON' 'IBM' 'INTC' 'JNJ' 'JPM' 'KO' 'MCD' 'MMM' 'MRK' 'MSFT' 'NKE' 'PG'\n",
      " 'TRV' 'UNH' 'V' 'VZ' 'WBA' 'WMT']\n",
      "Successfully transformed into array\n"
     ]
    }
   ],
   "source": [
    "train_env_config = get_train_env(TRAIN_START_DATE, VAL_END_DATE, \n",
    "                     ticker_list, data_source, time_interval, \n",
    "                        technical_indicator_list, env, model_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "pplgHdQtOOQH"
   },
   "source": [
    "## Registering for an environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FmSYCrWxGtam"
   },
   "outputs": [],
   "source": [
    "from ray.tune.registry import register_env\n",
    "env = StockTradingEnv_numpy\n",
    "env_name = 'StockTrading_train_env'\n",
    "register_env(env_name, lambda config: env(train_env_config))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6EAoWIYvs28E"
   },
   "source": [
    "##Defining hyperparameter search space\n",
    "\n",
    "1. You can look up [here](https://docs.ray.io/en/latest/tune/key-concepts.html#search-spaces) to learn how to define hyperparameter search space\n",
    "2. Jump over to this [link](https://github.com/DLR-RM/rl-baselines3-zoo/blob/master/utils/hyperparams_opt.py) to find the range of different hyperparameter\n",
    "3. To learn about different hyperparameters for different algorithms for RLlib models, jump over to this [link](https://docs.ray.io/en/latest/rllib-algorithms.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "WU9l7ceO15G4"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def loguniform(low=0,high=1):\n",
    "  return np.exp(np.random.uniform(low,high))\n",
    "\n",
    "def sample_ddpg_params():\n",
    "  \n",
    "  return {\n",
    "  \"buffer_size\": tune.choice([int(1e4), int(1e5), int(1e6)]),\n",
    "  \"lr\": tune.loguniform(1e-5, 1),\n",
    "  \"train_batch_size\": tune.choice([32, 64, 128, 256, 512])\n",
    "  }\n",
    "def sample_a2c_params():\n",
    "  \n",
    "  return{\n",
    "       \"lambda\": tune.choice([0.1,0.3,0.5,0.7,0.9,1.0]),\n",
    "      \"entropy_coeff\": tune.loguniform(0.00000001, 0.1),\n",
    "      \"lr\": tune.loguniform(1e-5, 1)    \n",
    "  }\n",
    "\n",
    "def sample_ppo_params():\n",
    "  return {\n",
    "     'lr':tune.loguniform(5e-5,0.0001),\n",
    "      'sgd_minibatch_size': tune.choice([64,128,256]),\n",
    "      'entropy_coeff': tune.loguniform(0.00000001, 0.1),\n",
    "        'clip_param':tune.choice([0.1,0.2,0.3,0.4]),\n",
    "        \"vf_loss_coeff\": tune.uniform(0,1),\n",
    "        'lambda': tune.choice([0.9, 0.95, 0.98, 0.99, 0.995,0.999]),\n",
    "        'kl_target': tune.choice([0.001,0.01,0.1]),\n",
    "  }\n",
    "\n",
    "\n",
    "def mutate_ddpg_hyperparams():\n",
    "  \n",
    "  return {\n",
    "  \"buffer_size\": [int(1e4), int(1e5), int(1e6)],\n",
    "  \"lr\": lambda: loguniform(1e-5, 1),\n",
    "  \"train_batch_size\": [32, 64, 128, 256, 512]\n",
    "  }\n",
    "\n",
    "def mutate_ppo_hyperparams():\n",
    "\n",
    "  return {\n",
    "    \"entropy_coeff\": lambda: loguniform(0.00000001, 0.1),\n",
    "      \"lr\": lambda: loguniform(5e-5, 0.0001),\n",
    "      \"sgd_minibatch_size\": [ 32, 64, 128, 256, 512],\n",
    "      \"lambda\": [0.9, 0.95, 0.98, 0.99, 0.995,0.999],\n",
    "      'clip_param': [0.1,0.2,0.3,0.4],\n",
    "      \"vf_loss_coeff\": lambda: np.random.uniform(0,1),\n",
    "      'kl_target': [0.001,0.01,0.1]\n",
    "\n",
    "  }\n",
    "  \n",
    "def mutate_a2c_hyperparams():\n",
    "  return{\n",
    "      \"lambda\": [0.1,0.3,0.5,0.7,0.9,1.0],\n",
    "    \"entropy_coeff\":  lambda: loguniform(0.00000001, 0.1),\n",
    "    \"lr\":  lambda: loguniform(1e-5, 1) \n",
    "}\n",
    "\n",
    "def a2c_params_bounds():\n",
    "  return {\n",
    "  \"buffer_size\": [int(1e4), int(1e6)],\n",
    "  \"lr\": [1e-5, 1],\n",
    "  \"train_batch_size\": [32, 512]\n",
    "  }\n",
    "\n",
    "def ppo_param_bounds():\n",
    "  return {\n",
    "    'lr':[5e-5, 0.0001],\n",
    "    'sgd_minibatch_size': [64,256],\n",
    "    'entropy_coeff': [0.00000001, 0.1],\n",
    "      'clip_param': [0.1,0.4],\n",
    "      \"vf_loss_coeff\": [0,1],\n",
    "      'lambda': [0.9,0.999],\n",
    "      'kl_target': [0.001,0.1], \n",
    "  }\n",
    "\n",
    "def ddpg_param_bounds():\n",
    "  return {\n",
    "    \"buffer_size\": [int(1e4), int(1e6)],\n",
    "    \"lr\": [1e-5, 1],\n",
    "    \"train_batch_size\": [32, 512]\n",
    "  }\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-yyeGzxN50v8"
   },
   "source": [
    "# Running Tune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "d2ppTjv-R870"
   },
   "outputs": [],
   "source": [
    "MODEL_TRAINER = {'a2c':A2CTrainer,'ppo':PPOTrainer,'ddpg':DDPGTrainer}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "W411kWlHdskQ"
   },
   "outputs": [],
   "source": [
    "num_samples = 1 #Population size\n",
    "training_iterations = 1 #Number of times the metric has been reported"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "VLDTWtZEqkU-"
   },
   "outputs": [],
   "source": [
    "if model_name == \"ddpg\":\n",
    "    sample_hyperparameters = sample_ddpg_params()\n",
    "    mutate_hyperparameters = mutate_ddpg_hyperparams()\n",
    "    params_bounds = ddpg_param_bounds()\n",
    "elif model_name == \"ppo\":\n",
    "  sample_hyperparameters = sample_ppo_params()\n",
    "  mutate_hyperparameters = mutate_ppo_hyperparams()\n",
    "  params_bounds = ppo_param_bounds()\n",
    "elif model_name == \"a2c\":\n",
    "  sample_hyperparameters = sample_a2c_params()\n",
    "  mutate_hyperparameters = mutate_a2c_hyperparams()\n",
    "  params_bounds = a2c_params_bounds()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "SbF64_hRsqhT"
   },
   "outputs": [],
   "source": [
    "\n",
    "pbt_scheduler = PopulationBasedTraining(\n",
    "    time_attr = \"training_iteration\",\n",
    "    perturbation_interval = training_iterations/5,\n",
    "    burn_in_period = 0.0,\n",
    "    quantile_fraction = 0.25,\n",
    "    hyperparam_mutations = mutate_hyperparameters\n",
    ")\n",
    "\n",
    "pb2_scheduler = PB2(\n",
    "                    time_attr='training_iteration',            \n",
    "                    perturbation_interval=training_iterations/5,\n",
    "                    quantile_fraction= 0.25,\n",
    "                    hyperparam_bounds={\n",
    "                       **params_bounds\n",
    "                    })\n",
    "\n",
    "def run_PBT(log_dir):\n",
    "\n",
    "  analysis = tune.run(\n",
    "      MODEL_TRAINER[model_name],\n",
    "      scheduler=pbt_scheduler, #To prune bad trials\n",
    "      metric='episode_reward_mean',\n",
    "      mode='max',\n",
    "      config = {**sample_hyperparameters,\n",
    "                'env':'StockTrading_train_env','num_workers':1,\n",
    "                'num_gpus':1,'framework':'torch','log_level':'DEBUG'},\n",
    "      num_samples = num_samples, #Number of hyperparameters to test out\n",
    "      stop = {'training_iteration':training_iterations},#Time attribute to validate the results\n",
    "      verbose=1,local_dir=\"./\"+log_dir,#Saving tensorboard plots\n",
    "      resources_per_trial={'gpu':1,'cpu':1},\n",
    "      max_failures = 1,#Extra Trying for the failed trials\n",
    "      raise_on_failed_trial=False,#Don't return error even if you have errored trials\n",
    "      # keep_checkpoints_num = 2, \n",
    "      checkpoint_score_attr ='episode_reward_mean',#Only store keep_checkpoints_num trials based on this score\n",
    "      checkpoint_freq=training_iterations,#Checpointing all the trials,\n",
    "  )\n",
    "  # print(\"Best hyperparameter: \", analysis.best_config)\n",
    "  return analysis\n",
    "\n",
    "def run_PB2(log_dir):\n",
    "\n",
    "  analysis = tune.run(\n",
    "      MODEL_TRAINER[model_name],\n",
    "      scheduler=pb2_scheduler, #To prune bad trials\n",
    "      metric='episode_reward_mean',\n",
    "      mode='max',\n",
    "      config = {**sample_hyperparameters,\n",
    "                'env':'StockTrading_train_env','num_workers':1,\n",
    "                'num_gpus':1,'framework':'torch','log_level':'DEBUG'},\n",
    "      num_samples = num_samples, #Number of hyperparameters to test out\n",
    "      stop = {'training_iteration':training_iterations},#Time attribute to validate the results\n",
    "      verbose=1,local_dir=\"./\"+log_dir,#Saving tensorboard plots\n",
    "      resources_per_trial={'gpu':1,'cpu':1},\n",
    "      max_failures = 1,#Extra Trying for the failed trials\n",
    "      raise_on_failed_trial=False,#Don't return error even if you have errored trials\n",
    "      # keep_checkpoints_num = 2, \n",
    "      checkpoint_score_attr ='episode_reward_mean',#Only store keep_checkpoints_num trials based on this score\n",
    "      checkpoint_freq=training_iterations,#Checpointing all the trials\n",
    "  # print(\"Best hyperparameter: \", analysis.best_config)\n",
    "  return analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zDz4GUMLuSUE"
   },
   "outputs": [],
   "source": [
    "# analysis = run_PB2(\"PB2_Tuned\")\n",
    "analysis = run_PB2(\"PB2_Tuned\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "mJ4kxzsKK__h"
   },
   "outputs": [],
   "source": [
    "# !zip -r /content/PBT_Tuned.zip /content/PBT_Tuned\n",
    "# from google.colab import files\n",
    "# files.download(\"/content/PBT_Tuned.zip\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6d3a8-KROYJ_"
   },
   "source": [
    "## Best config, hyperparameters and checkpoints\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "OGDP01DcCR9Z",
    "outputId": "f7372ddd-fdee-4bd8-dd60-a92052c168ae"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'clip_param': 0.1,\n",
       " 'entropy_coeff': 2.7984540504890533e-08,\n",
       " 'env': 'StockTrading_train_env',\n",
       " 'framework': 'torch',\n",
       " 'kl_target': 0.1,\n",
       " 'lambda': 0.95,\n",
       " 'log_level': 'DEBUG',\n",
       " 'lr': 8.016314885885411e-05,\n",
       " 'num_gpus': 1,\n",
       " 'num_workers': 1,\n",
       " 'sgd_minibatch_size': 64,\n",
       " 'vf_loss_coeff': 0.2489739417927158}"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_config = analysis.get_best_config(metric='episode_reward_mean',mode='max')\n",
    "best_config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 53
    },
    "id": "Awbo9S2sZbOv",
    "outputId": "0144ec83-423a-4a5e-902b-b51882ca892f"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'/content/PB2_Tuned/PPOTrainer_2022-07-10_21-59-13/PPOTrainer_StockTrading_train_env_89281_00000_0_clip_param=0.1000,entropy_coeff=0.0000,kl_target=0.1000,lambda=0.9500,lr=0.0001,sg_2022-07-10_21-59-15'"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_logdir = analysis.get_best_logdir(metric='episode_reward_mean',mode='max')\n",
    "best_logdir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "wa-dilLhHGEd"
   },
   "outputs": [],
   "source": [
    "best_checkpoint = analysis.best_checkpoint"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0qNIyaSYfofv"
   },
   "source": [
    "### TESTING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "MyXiBe1k4yIq"
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "!unzip \"/content/PB2_Tuned.zip\"\n",
    "!unzip \"/content/PBT_Tuned.zip\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "eI5nFT-Z39SS",
    "outputId": "d32de84f-dbda-4270-f002-857609a336bd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "Shape of DataFrame:  (8820, 9)\n",
      "Clean data for AAPL\n",
      "Data clean for AAPL is finished.\n",
      "Clean data for AMGN\n",
      "Data clean for AMGN is finished.\n",
      "Clean data for AXP\n",
      "Data clean for AXP is finished.\n",
      "Clean data for BA\n",
      "Data clean for BA is finished.\n",
      "Clean data for CAT\n",
      "Data clean for CAT is finished.\n",
      "Clean data for CRM\n",
      "Data clean for CRM is finished.\n",
      "Clean data for CSCO\n",
      "Data clean for CSCO is finished.\n",
      "Clean data for CVX\n",
      "Data clean for CVX is finished.\n",
      "Clean data for DIS\n",
      "Data clean for DIS is finished.\n",
      "Clean data for DOW\n",
      "Data clean for DOW is finished.\n",
      "Clean data for GS\n",
      "Data clean for GS is finished.\n",
      "Clean data for HD\n",
      "Data clean for HD is finished.\n",
      "Clean data for HON\n",
      "Data clean for HON is finished.\n",
      "Clean data for IBM\n",
      "Data clean for IBM is finished.\n",
      "Clean data for INTC\n",
      "Data clean for INTC is finished.\n",
      "Clean data for JNJ\n",
      "Data clean for JNJ is finished.\n",
      "Clean data for JPM\n",
      "Data clean for JPM is finished.\n",
      "Clean data for KO\n",
      "Data clean for KO is finished.\n",
      "Clean data for MCD\n",
      "Data clean for MCD is finished.\n",
      "Clean data for MMM\n",
      "Data clean for MMM is finished.\n",
      "Clean data for MRK\n",
      "Data clean for MRK is finished.\n",
      "Clean data for MSFT\n",
      "Data clean for MSFT is finished.\n",
      "Clean data for NKE\n",
      "Data clean for NKE is finished.\n",
      "Clean data for PG\n",
      "Data clean for PG is finished.\n",
      "Clean data for TRV\n",
      "Data clean for TRV is finished.\n",
      "Clean data for UNH\n",
      "Data clean for UNH is finished.\n",
      "Clean data for V\n",
      "Data clean for V is finished.\n",
      "Clean data for VZ\n",
      "Data clean for VZ is finished.\n",
      "Clean data for WBA\n",
      "Data clean for WBA is finished.\n",
      "Clean data for WMT\n",
      "Data clean for WMT is finished.\n",
      "Data clean all finished!\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "Shape of DataFrame:  (294, 9)\n",
      "Clean data for ^VIX\n",
      "Data clean for ^VIX is finished.\n",
      "Data clean all finished!\n",
      "['AAPL' 'AMGN' 'AXP' 'BA' 'CAT' 'CRM' 'CSCO' 'CVX' 'DIS' 'DOW' 'GS' 'HD'\n",
      " 'HON' 'IBM' 'INTC' 'JNJ' 'JPM' 'KO' 'MCD' 'MMM' 'MRK' 'MSFT' 'NKE' 'PG'\n",
      " 'TRV' 'UNH' 'V' 'VZ' 'WBA' 'WMT']\n",
      "Successfully transformed into array\n"
     ]
    }
   ],
   "source": [
    "test_config = get_test_config(TEST_START_DATE, TEST_END_DATE, ticker_list, data_source, time_interval, \n",
    "         technical_indicator_list, env, model_name, if_vix = True)\n",
    "\n",
    "test_env_name = 'StockTrading_test_env'\n",
    "register_env(env_name, lambda config: env(test_config))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "95xsCNoAmlwW"
   },
   "outputs": [],
   "source": [
    "# import json\n",
    "# chkpt_dir = \"/content/content/PB2_Tuned/PPOTrainer_2022-06-30_18-03-49/PPOTrainer_StockTrading_train_env_fe24f_00006_6_clip_param=0.2000,entropy_coeff=0.0323,kl_target=0.1000,lambda=0.9950,lr=0.0001,sg_2022-06-30_18-09-42\"\n",
    "\n",
    "# with open(chkpt_dir+'/params.json') as f:\n",
    "#   model_config = json.load(f)\n",
    "\n",
    "# model_config['env'] = test_env_name\n",
    "# print(model_config)\n",
    "# trainer = ppo.PPOTrainer(config=model_config)\n",
    "# trainer.restore(chkpt_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "F26_qHGn5819"
   },
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4T1LxPZf67ja"
   },
   "outputs": [],
   "source": [
    "def DRL_prediction(\n",
    "        model_name:str,\n",
    "        test_env_config:dict,\n",
    "        env:StockTradingEnv_numpy,\n",
    "        model_config:dict,\n",
    "        agent_path:str,\n",
    "        env_name_test='StockTrading_test_env'\n",
    "    ):\n",
    "\n",
    "        env_instance = env(test_env_config)\n",
    "        \n",
    "        register_env(env_name_test, lambda config: env(test_env_config))\n",
    "        model_config['env'] = env_name_test\n",
    "        # ray.init() # Other Ray APIs will not work until `ray.init()` is called.\n",
    "        if model_name == \"ppo\":\n",
    "            trainer = MODELS[model_name].PPOTrainer(config=model_config)\n",
    "        elif model_name == \"a2c\":\n",
    "            trainer = MODELS[model_name].A2CTrainer(config=model_config)\n",
    "        elif model_name == \"ddpg\":\n",
    "            trainer = MODELS[model_name].DDPGTrainer(config=model_config)\n",
    "        elif model_name == \"td3\":\n",
    "            trainer = MODELS[model_name].TD3Trainer(config=model_config)\n",
    "        elif model_name == \"sac\":\n",
    "            trainer = MODELS[model_name].SACTrainer(config=model_config)\n",
    "\n",
    "        try:\n",
    "            trainer.restore(agent_path)\n",
    "            print(\"Restoring from checkpoint path\", agent_path)\n",
    "        except BaseException:\n",
    "            raise ValueError(\"Fail to load agent!\")\n",
    "\n",
    "        # test on the testing env\n",
    "        state = env_instance.reset()\n",
    "        episode_returns = list()  # the cumulative_return / initial_account\n",
    "        episode_total_assets = list()\n",
    "        episode_total_assets.append(env_instance.initial_total_asset)\n",
    "        done = False\n",
    "        while not done:\n",
    "            action = trainer.compute_single_action(state)\n",
    "            state, reward, done, _ = env_instance.step(action)\n",
    "\n",
    "            total_asset = (\n",
    "                env_instance.amount\n",
    "                + (env_instance.price_ary[env_instance.day] * env_instance.stocks).sum()\n",
    "            )\n",
    "            episode_total_assets.append(total_asset)\n",
    "            episode_return = total_asset / env_instance.initial_total_asset\n",
    "            episode_returns.append(episode_return)\n",
    "        ray.shutdown()\n",
    "        print(\"episode return: \" + str(episode_return))\n",
    "        print(\"Test Finished!\")\n",
    "        return episode_total_assets\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3GIejkts2SEL",
    "outputId": "1348c973-0813-4ff4-f809-86ec087d3e39"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-10 22:03:58,436\tINFO ppo.py:415 -- In multi-agent mode, policies will be optimized sequentially by the multi-GPU optimizer. Consider setting simple_optimizer=True if this doesn't work for you.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m 2022-07-10 22:04:12,767\tWARNING env.py:136 -- Your env doesn't have a .spec.max_episode_steps attribute. This is fine if you have set 'horizon' in your config dictionary, or `soft_horizon`. However, if you haven't, 'horizon' will default to infinity, and your environment will not be reset.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m 2022-07-10 22:04:12,781\tDEBUG rollout_worker.py:1770 -- Creating policy for default_policy\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m 2022-07-10 22:04:12,789\tDEBUG catalog.py:807 -- Created preprocessor <ray.rllib.models.preprocessors.NoPreprocessor object at 0x7fdd576afe50>: Box([-3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  -3000. -3000. -3000.], [3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m  3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.], (333,), float32) -> (333,)\n",
      "2022-07-10 22:04:12,908\tINFO worker_set.py:163 -- Inferred observation/action spaces from remote worker (local worker has no env): {'default_policy': (Box([-3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000.], [3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.], (333,), float32), Box([-1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1.\n",
      " -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1.], [1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n",
      " 1. 1. 1. 1. 1. 1.], (30,), float32)), '__env__': (Box([-3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000.], [3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.], (333,), float32), Box([-1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1.\n",
      " -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1.], [1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n",
      " 1. 1. 1. 1. 1. 1.], (30,), float32))}\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m 2022-07-10 22:04:12,827\tINFO torch_policy.py:193 -- TorchPolicy (worker=1) running on CPU.\n",
      "\u001B[2m\u001B[36m(RolloutWorker pid=8554)\u001B[0m 2022-07-10 22:04:12,871\tDEBUG rollout_worker.py:785 -- Created rollout worker with env <ray.rllib.env.vector_env.VectorEnvWrapper object at 0x7fdd576bff10> (<StockTradingEnv instance>), policies {}\n",
      "2022-07-10 22:04:13,013\tDEBUG rollout_worker.py:1770 -- Creating policy for default_policy\n",
      "2022-07-10 22:04:13,047\tDEBUG catalog.py:807 -- Created preprocessor <ray.rllib.models.preprocessors.NoPreprocessor object at 0x7f2609c97490>: Box([-3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000. -3000.\n",
      " -3000. -3000. -3000.], [3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.\n",
      " 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000. 3000.], (333,), float32) -> (333,)\n",
      "2022-07-10 22:04:13,073\tINFO torch_policy.py:215 -- TorchPolicy (worker=local) running on 1 GPU(s).\n",
      "2022-07-10 22:04:28,285\tINFO rollout_worker.py:1793 -- Built policy map: {}\n",
      "2022-07-10 22:04:28,292\tINFO rollout_worker.py:1794 -- Built preprocessor map: {'default_policy': <ray.rllib.models.preprocessors.NoPreprocessor object at 0x7f2609c97490>}\n",
      "2022-07-10 22:04:28,296\tINFO rollout_worker.py:670 -- Built filter map: {'default_policy': <ray.rllib.utils.filter.NoFilter object at 0x7f2609d9d5d0>}\n",
      "2022-07-10 22:04:28,302\tDEBUG rollout_worker.py:785 -- Created rollout worker with env None (None), policies {}\n",
      "2022-07-10 22:04:28,315\tINFO trainable.py:163 -- Trainable.setup took 29.894 seconds. If your trainable is slow to initialize, consider setting reuse_actors=True to reduce actor creation overheads.\n",
      "2022-07-10 22:04:28,518\tINFO trainable.py:589 -- Restored on 172.28.0.2 from checkpoint: /content/PB2_Tuned/PPOTrainer_2022-07-10_21-59-13/PPOTrainer_StockTrading_train_env_89281_00000_0_clip_param=0.1000,entropy_coeff=0.0000,kl_target=0.1000,lambda=0.9500,lr=0.0001,sg_2022-07-10_21-59-15/checkpoint_000001/checkpoint-1\n",
      "2022-07-10 22:04:28,528\tINFO trainable.py:597 -- Current state after restoring: {'_iteration': 1, '_timesteps_total': None, '_time_total': 54.82635021209717, '_episodes_total': 13}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Restoring from checkpoint path /content/PB2_Tuned/PPOTrainer_2022-07-10_21-59-13/PPOTrainer_StockTrading_train_env_89281_00000_0_clip_param=0.1000,entropy_coeff=0.0000,kl_target=0.1000,lambda=0.9500,lr=0.0001,sg_2022-07-10_21-59-15/checkpoint_000001/checkpoint-1\n",
      "episode return: 1.139153132802156\n",
      "Test Finished!\n"
     ]
    }
   ],
   "source": [
    "episode_total_assets = DRL_prediction(\n",
    "        model_name,\n",
    "        test_config,\n",
    "        env,\n",
    "        best_config,\n",
    "        '/content/PB2_Tuned/PPOTrainer_2022-07-10_21-59-13/PPOTrainer_StockTrading_train_env_89281_00000_0_clip_param=0.1000,entropy_coeff=0.0000,kl_target=0.1000,lambda=0.9500,lr=0.0001,sg_2022-07-10_21-59-15/checkpoint_000001/checkpoint-1',\n",
    "        env_name_test='StockTrading_test_env')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Q_kxAWHHJAP0",
    "outputId": "94ac767e-baec-40dd-e690-670deaf460df"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7742295439858137"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calculate_sharpe(episode_total_assets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2ULXtSmlJWDd"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "FinRL_PBT PB2 _Raytune_for_Hyperparameter_Optimization.ipynb",
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
